/*
 * Copyright 2008 Carnegie Mellon University
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *  
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, 
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package edu.cmu.lti.sepia.servlet;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;

import edu.cmu.lti.sepia.data.db.AnswerDAO;
import edu.cmu.lti.sepia.data.db.NuggetDAO;
import edu.cmu.lti.sepia.data.db.SQL;
import edu.cmu.lti.sepia.data.db.TopicDAO;
import edu.cmu.lti.sepia.infra.topic.GoldStandardXmlFacade;
import edu.cmu.lti.sepia.infra.topic.model.Answer;
import edu.cmu.lti.sepia.infra.topic.model.Nugget;
import edu.cmu.lti.sepia.infra.topic.model.Topic;
import edu.cmu.lti.sepia.infra.topic.model.TopicSet;
import edu.cmu.lti.sepia.util.ExceptionUtil;
import edu.cmu.lti.sepia.util.Logger;
import edu.cmu.lti.sepia.util.ServletUtil;

/**
 * Servlet that allows a user to import topics with answers. 
 * 
 * @author Hideki Shima
 *
 */
public class TopicImporter extends HttpServlet {

  private static final long serialVersionUID = 1L;

  private final static int THRESHOLD = 50*1024*1024;
  private final static long MAXSIZE = 100*1024*1024;
  
  private final static String formAction = "/io/topic_importer.html";

  private final static String UPLOAD_SUCCEEDED = "<br><strong style=\"color:#ff0000\">Upload succeeded.</strong><br>\n";
  private final static String UPLOAD_SYSTEM_ERROR = "<br><strong style=\"color:#ff0000\">Upload failed. Please report the error message and send the file to the system admin by email.</strong><br>\n";
  
  public void doGet(HttpServletRequest request, HttpServletResponse response)
    throws ServletException, IOException {

    ServletOutputStream out = response.getOutputStream();
    Commons.printHeader(request, response, "Topic Importer", 2);
    out.println( getUploadForm(request, response, request.getContextPath()+formAction, "") );
    Commons.printFooter(request, response);
  }
  
  public void doPost(HttpServletRequest request, HttpServletResponse response)
	  throws ServletException, IOException {
    ServletOutputStream out = response.getOutputStream();
    Commons.printHeader(request, response, "Topic Importer", 2);
    
    ServletUtil s = new ServletUtil( request );
    String taskNameShort = s.readString("taskNameShort", "");
    
    SQL sql = new SQL();
    try {
      String report = upload( request, response, sql, taskNameShort );
      out.println( "<hr>" );
      out.println( report );
      out.println( getUploadForm(request, response, request.getContextPath()+formAction, "") );
    } catch (Exception e) {
      throw new IOException(e);
    } finally {
      try {
        sql.close();
      } catch (Exception e){e.printStackTrace();}
    }
    
    Commons.printFooter(request, response);
	}
  
  public String getUploadForm(HttpServletRequest request,
          HttpServletResponse response, String actionPath, String filetype) 
    throws IOException {

    StringBuilder sb = new StringBuilder();
    sb.append("<br><h4>Data File Uploader</h4>\n");
    sb.append("Select the file type, specifiy the file and then click submit.<br><br>");
    
    sb.append("<form action=\""+actionPath+"\" method=\"POST\" enctype=\"multipart/form-data\" onsubmit=\"if(!$('file').value){alert('Please select the file to upload');return false;};\">\n");    
    sb.append("<table>\n");
    sb.append("<tr><td>File type</td><td>\n");
    sb.append("<input type=\"radio\" name=\"filetype\" id=\"tsv\" value=\"tsv\""
      +(filetype.equalsIgnoreCase("tsv")?" checked":"")+"><label id=\"tsv_help\" for=\"tsv\" title=\"TSV format (each line contains topic id and question in target language separated by a tab or spaces; e.g. 'Topic001\tWhat is ...?')\">TSV</label>&nbsp;&nbsp;\n");
    sb.append("<input type=\"radio\" name=\"filetype\" id=\"xml\" value=\"xml\""
      +(filetype.equalsIgnoreCase("xml")?" checked":"")+"><label id=\"xml_help\" for=\"xml\" title=\"SEPIA's XML format (i.e. <TOPIC_SET><TOPIC>...)\">XML</label>\n");
    sb.append("<!--Filetype = "+filetype+"-->\n");
    sb.append("</td></tr>\n");
    sb.append("<tr><td>File</td><td><input type=\"file\" name=\"file\" id=\"file\"></td></tr>\n");
    sb.append("<tr><td colspan=2><input type=\"submit\" value=\" submit \"></td></tr>\n");
    sb.append("</table>");
    sb.append("</form>\n");
    sb.append("<script type=\"text/javascript\">\n");
    sb.append("jQuery(document).ready(function($) {\n");
    sb.append("  $('#tsv_help,#xml_help').tipsy({gravity: 's'}); });\n");
    sb.append("</script>\n");
    return sb.toString();
  }

  @SuppressWarnings("unchecked")
  public String upload(HttpServletRequest request, HttpServletResponse response, 
          SQL sql, String taskNameShort ) throws ServletException, IOException {

    ServletUtil s = new ServletUtil( request );
    String srcLang = s.readString("srcLang", "EN");
    String trgLang = s.readString("trgLang", "EN");
    
    StringBuilder report = new StringBuilder();
    InputStream is = null;
    
    try {
      DiskFileItemFactory factory = new DiskFileItemFactory();
      factory.setSizeThreshold(THRESHOLD);
      
      ServletFileUpload upload = new ServletFileUpload(factory);
      upload.setFileSizeMax(MAXSIZE);
      upload.setHeaderEncoding("utf-8");
      
      List<FileItem> items = (List<FileItem>) upload.parseRequest(request);
      
      List<Topic> topics = TopicDAO.getTopics(sql, taskNameShort, null, -1);
      Map<String,Topic> topicMap = new HashMap<String,Topic>();
      for ( Topic t : topics ) {
        topicMap.put(t.getExternalId(), t);
      }
      
      Map<String,String> formParams = new HashMap<String,String>();
      String content = "";
      for (FileItem item : items) {
        if (!item.isFormField()) {
          is = item.getInputStream();
          StringBuilder sb = new StringBuilder();
          InputStreamReader isr = new InputStreamReader(is);
          BufferedReader br = new BufferedReader(isr);
          String line = null;
          while ( (line = br.readLine())!=null ) {
            sb.append(line+"\n");
          }
          content = sb.toString();
        } else {
          formParams.put( item.getFieldName(), item.getString() );
        }
      }
      
      String filetype = formParams.get("filetype"); // mawarikudoi way of getting the value
      List<Topic> newTopics = new ArrayList<Topic>();
      if ( filetype.equalsIgnoreCase("tsv") ) {
        newTopics = importTopicsFromTSV( content, topicMap );
      } else if ( filetype.equalsIgnoreCase("xml") ) {
//        out.println("<pre>"+content+"</pre>");
        newTopics = loadTopicFromXML( content, topicMap, srcLang, trgLang );
      }
      
      report.append("<br>Affected "+newTopics.size()+" topics.<br><ul>\n");
      for ( Topic topic : newTopics ) {
        int topicId = TopicDAO.findInternalIdByExternalId(sql, taskNameShort, topic.getExternalId());
        if ( topicId==-1 ) {
          synchronized ( topic ) {
            TopicDAO.addTopic( sql, taskNameShort, topic );
            topicId = TopicDAO.findLatest(sql, taskNameShort).getTopicId();            
          }
          report.append("<li>Added a new topic (external id = "+topic.getExternalId()+")");
        } else {
          TopicDAO.updateTopic( sql, taskNameShort, topic );
          report.append("<li>Updated a topic (internal id = "+topic.getTopicId()+", external id = "+topic.getExternalId()+")");
        }
        List<Answer> answers = topic.getAnswers();
        for ( Answer a : answers ) {
          a.setTopicId(topicId);
        }
        if (answers!=null) {
          report.append(" with "+answers.size()+" answer texts");
        }
        List<Nugget> nuggets = topic.getNuggets();
        for ( Nugget n : nuggets ) {
          n.setTopicId(topicId);
        }
        if (nuggets!=null) {
          report.append(" and "+nuggets.size()+" answer nuggets");
        }
        report.append(".\n");
        AnswerDAO.addAnswers( sql, taskNameShort, answers );
        NuggetDAO.addNuggets( sql, taskNameShort, nuggets );
      }
      report.append("</ul>");
      
      report.append(UPLOAD_SUCCEEDED);
      Logger.log(sql, request.getSession(), "UPLOAD TOPICS", newTopics.size()+" topics");
    } catch (Exception e) {
      report.append(UPLOAD_SYSTEM_ERROR+"<br>"+ExceptionUtil.exceptionToString(e));
    } finally {
      if (is!=null ) {
        is.close();
      }
    }

    return report.toString();
  }
  
  private Pattern pTSV = Pattern.compile("^([^\\s]+)\\s+(.+)$");
  private List<Topic> importTopicsFromTSV( String content,  Map<String,Topic> topicMap ) {
    List<Topic> topics = new ArrayList<Topic>();
    String[] lines = content.split("\\n");
    for ( String line : lines ) {
      Matcher mTSV = pTSV.matcher(line);
      if (mTSV.find()) {
        String tid = mTSV.group(1);
        String question = mTSV.group(2).trim();
        Topic t0 = topicMap.get(tid);
        if ( t0!=null ) {
          t0.setQuestionTrg(question);
        } else {
          t0 = new Topic(-1, tid, "", "", question, "", "", "", "");
        }
        topics.add(t0);
      }
    }
    return topics;
  }
  
  private List<Topic> loadTopicFromXML( String content, Map<String,Topic> topicMap, 
          String srcLang, String trgLang ) {
    List<Topic> topics = new ArrayList<Topic>();
    
    TopicSet ts = GoldStandardXmlFacade.deserializeTopicSet(content, srcLang, trgLang);
    
    for ( Topic t1 : ts.values() ) { // new topic
      String tid = t1.getExternalId();
      Topic t0 = topicMap.get(tid); // existing topic
      if ( t0!=null ) { // overwrite if new value found
        String a = t1.getAnswerType();
        if (a!=null&&a.length()>0) t0.setAnswerType(a);
        String t = t1.getTitle();
        if (t!=null&&t.length()>0) t0.setTitle(t);
        String qs = t1.getQuestionSrc();
        if (qs!=null&&qs.length()>0) t0.setQuestionSrc(qs);
        String qt = t1.getQuestionTrg();
        if (qt!=null&&qt.length()>0) t0.setQuestionTrg(qt);
        String ns = t1.getNarrativeSrc();
        if (ns!=null&&ns.length()>0) t0.setNarrativeSrc(ns);
        String nt = t1.getNarrativeTrg();
        if (nt!=null&&nt.length()>0) t0.setNarrativeTrg(nt);
        String n = t1.getNote();
        if (n!=null&&n.length()>0) t0.setNote(n);
        List<Answer> answers = t1.getAnswers();
        if (answers!=null) t0.setAnswers(answers);
        List<Nugget> nuggets = t1.getNuggets();
        if (nuggets!=null) t0.setNuggets(nuggets);
        topics.add(t0);
      } else {
        topics.add(t1);
      }
    }
    return topics;
  }
  
}
